{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Various torch packages\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# torchvision\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# ------------------------\n",
    "# get up one directory \n",
    "import sys, os\n",
    "sys.path.append(os.path.abspath('../'))\n",
    "# ------------------------\n",
    "\n",
    "# custom packages\n",
    "import models.aux_funs as maf\n",
    "import optimizers as op\n",
    "import regularizers as reg\n",
    "import train\n",
    "import math\n",
    "import utils.configuration as cf\n",
    "import utils.datasets as ud\n",
    "from utils.datasets import get_data_set, GaussianSmoothing\n",
    "from models.fully_connected import fully_connected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# Fix random seed\n",
    "# -----------------------------------------------------------------------------------\n",
    "random_seed = 3\n",
    "cf.seed_torch(random_seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test cases\n",
    "## Test case 0: No Skips, Denoising\n",
    "* No skips (regularization parameter for skips is set to a large value)\n",
    "* Denoising task, no blur is added to the images\n",
    "\n",
    "## Test case 1: No Skips, Deblurring\n",
    "* No skips (regularization parameter for skips is set to a large value)\n",
    "* Deblurring task, blur is added to the images\n",
    "\n",
    "## Test case 2: Skips, Denoising\n",
    "* Skips are allowed but regularized\n",
    "* Denoising task, no blur is added to the images\n",
    "\n",
    "## Test case 3: Skips, Deblurring\n",
    "* Skips are allowed but regularized\n",
    "* Deblurring task, blur is added to the images "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_case = 0\n",
    "\n",
    "if test_case == 0:\n",
    "    lamda_0 = 0.07\n",
    "    lamda_1 = 1e10\n",
    "    add_blur = False\n",
    "elif test_case == 1:\n",
    "    lamda_0 = 0.07\n",
    "    lamda_1 = 1e10\n",
    "    add_blur = True\n",
    "elif test_case == 2:\n",
    "    lamda_0 = 0.07\n",
    "    lamda_1 = 28*lamda_0\n",
    "    add_blur = False\n",
    "elif test_case == 3:\n",
    "    lamda_0 = 0.07\n",
    "    lamda_1 = 28*lamda_0\n",
    "    add_blur = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def reshaped_mse_loss(x,y):\n",
    "    return torch.nn.MSELoss()(x,y.view(-1,28*28))\n",
    "# -----------------------------------------------------------------------------------\n",
    "# Parameters\n",
    "# -----------------------------------------------------------------------------------\n",
    "conf_args = {#\n",
    "    # data specification\n",
    "    'data_file':\"../../Data\", 'train_split':0.95, 'data_set':\"Encoder-MNIST\", 'download':False,\n",
    "    'add_noise':True, 'noise_std':0.05, 'add_blur':add_blur,\n",
    "    # cuda\n",
    "    'use_cuda':True, 'num_workers':4, 'cuda_device':0, 'pin_memory':False,\n",
    "    #\n",
    "    'epochs':100, 'loss':reshaped_mse_loss,\n",
    "    # optimizer\n",
    "    'delta':1.0, 'lr':0.001, 'lamda_0': lamda_0, 'lamda_1':lamda_1, 'optim':\"AdaBreg\", 'row_group':True,\n",
    "    'reg':reg.reg_l1_l2, 'beta':0.0,\n",
    "    # model\n",
    "    'model_size':7*[28*28], 'act_fun':torch.nn.ReLU(),\n",
    "    # initialization\n",
    "    'sparse_init':0.01, 'r':[1,5,1],\n",
    "    # misc\n",
    "    'random_seed':random_seed, 'eval_acc':False, 'name':'main-Encoder-MNIST', 'super_type':'Encoder'\n",
    "}\n",
    "conf = cf.Conf(**conf_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# General ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicBlock(nn.Module):\n",
    "    def __init__(self, in_neurons, out_neurons, idx, act_fun, act_fun_outer):\n",
    "        super(BasicBlock, self).__init__()\n",
    "        self.act_fun = act_fun\n",
    "        self.act_fun_outer = act_fun_outer\n",
    "        self.linear = nn.Linear(in_neurons, out_neurons)\n",
    "\n",
    "        s = torch.zeros((idx+1,))              \n",
    "                        \n",
    "        self.skips = nn.Parameter(s, requires_grad=True)\n",
    "        self.idx = idx  \n",
    "        \n",
    "    def forward(self, x):    \n",
    "        x2 = torch.sum(self.skips.view(1,1,-1) * x,2)\n",
    "        x3 = x[:,:,-1]\n",
    "        \n",
    "        out = self.act_fun_outer(self.act_fun(self.linear(x3)) + x2)\n",
    "        #\n",
    "        return torch.cat((x,out.unsqueeze(2)),2)\n",
    "    \n",
    "class OutBlock(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(OutBlock, self).__init__()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return x[:,:,-1]\n",
    "\n",
    "class fully_skip_connected(nn.Module):\n",
    "    def __init__(self, sizes, act_fn, outer_act_fn = nn.Identity(), mean=0.0, std=1.0):\n",
    "        super(fully_skip_connected, self).__init__()\n",
    "        self.mean = mean\n",
    "        self.std = std\n",
    "        self.num_l = len(sizes)\n",
    "        \n",
    "        self.act_fn = act_fn\n",
    "        layer_list = []\n",
    "        for i in range(self.num_l-1):\n",
    "            layer_list.append(BasicBlock(sizes[i], sizes[i+1], i, self.act_fn, outer_act_fn))\n",
    "            \n",
    "        layer_list.append(OutBlock())\n",
    "            \n",
    "        self.layers = nn.Sequential(*layer_list)\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = (x-self.mean)/self.std\n",
    "        x = nn.Flatten()(x)\n",
    "        x = x.unsqueeze(2)\n",
    "        x = self.layers(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# define the model and an instance of the best model class\n",
    "# -----------------------------------------------------------------------------------\n",
    "model_kwargs = {'mean':conf.data_set_mean, 'std':conf.data_set_std}    \n",
    "\n",
    "model_conf = \"fsc\"\n",
    "if model_conf == \"fsc\":\n",
    "    model = fully_skip_connected(conf.model_size, conf.act_fun, **model_kwargs)\n",
    "    best_model = train.best_model(fully_skip_connected(conf.model_size, conf.act_fun, **model_kwargs).to(conf.device))\n",
    "elif model_conf == \"fc\":\n",
    "    model = fully_connected(conf.model_size, conf.act_fun, **model_kwargs)\n",
    "    best_model = train.best_model(fully_connected(conf.model_size, conf.act_fun, **model_kwargs).to(conf.device))\n",
    "    \n",
    "# sparsify\n",
    "maf.sparse_bias_uniform_(model, 0,conf.r[0])\n",
    "maf.sparse_weight_normal_(model, conf.r[1])\n",
    "maf.sparsify_(model, conf.sparse_init, row_group = conf.row_group)\n",
    "model = model.to(conf.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# define the model and an instance of the best model class\n",
    "# -----------------------------------------------------------------------------------\n",
    "model_kwargs = {'mean':conf.data_set_mean, 'std':conf.data_set_std}    \n",
    "\n",
    "def init_weights(conf, model):\n",
    "    # sparsify\n",
    "    maf.sparse_bias_uniform_(model, 0, conf.r[0])\n",
    "    maf.sparse_weight_normal_(model, conf.r[1])\n",
    "    maf.sparsify_(model, conf.sparse_init, row_group = conf.row_group)\n",
    "    model = model.to(conf.device)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# Optimizer\n",
    "# -----------------------------------------------------------------------------------\n",
    "def get_skips(model):\n",
    "    for m in model.modules():\n",
    "        if hasattr(m,'skips'):\n",
    "            yield m.skips\n",
    "        else:\n",
    "            continue\n",
    "\n",
    "def print_skips(model):\n",
    "    for m in model.modules():\n",
    "        if hasattr(m,'skips'):\n",
    "            print((0.001*torch.round(1000*m.skips.data).cpu()))\n",
    "            \n",
    "def skips_to_list(model):\n",
    "    skips = []\n",
    "    for m in model.modules():\n",
    "        if hasattr(m,'skips'):\n",
    "            skips.append(m.skips.data.tolist())\n",
    "    return skips\n",
    "    \n",
    "\n",
    "def init_opt(conf, model):\n",
    "    # Get access to different model parameters\n",
    "    weights_linear = maf.get_weights_linear(model)\n",
    "    biases = maf.get_bias(model)\n",
    "    skips = get_skips(model)\n",
    "\n",
    "    # -----------------------------------------------------------------------------------\n",
    "    # Initialize optimizer\n",
    "    # -----------------------------------------------------------------------------------\n",
    "    reg_0 = conf.reg(lamda=conf.lamda_0)\n",
    "    reg_1 = reg.reg_l1(lamda=conf.lamda_1)\n",
    "\n",
    "    if conf.optim == \"SGD\":\n",
    "        opt = torch.optim.SGD(model.parameters(), lr=conf.lr, momentum=conf.beta)\n",
    "    elif conf.optim == \"AdaBreg\":\n",
    "        opt = op.AdaBreg([{'params': weights_linear, 'lr' : conf.lr, 'reg' : reg_0},\n",
    "                           {'params': biases, 'lr': conf.lr},\n",
    "                           {'params': skips, 'lr':conf.lr, 'reg':reg_1}])\n",
    "\n",
    "    # learning rate scheduler\n",
    "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, factor=0.5, patience=5,threshold=0.01)\n",
    "    \n",
    "    return opt, scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "save_params = False\n",
    "if save_params:\n",
    "    conf.write_to_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Program Files (x86)\\Python37\\lib\\site-packages\\torch\\functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\builder\\windows\\pytorch\\aten\\src\\ATen\\native\\TensorShape.cpp:3191.)\n",
      "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
     ]
    }
   ],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# Prepare Data\n",
    "# -----------------------------------------------------------------------------------\n",
    "smoothing = GaussianSmoothing(1, 9, 5.0)   \n",
    "transform = []\n",
    "\n",
    "if conf.add_blur:\n",
    "    transform.append(smoothing)\n",
    "\n",
    "if conf.add_noise:\n",
    "    transform.append(ud.add_noise(std=conf.noise_std,device='cpu'))\n",
    "    \n",
    "transform = transforms.Compose(transform)   \n",
    "\n",
    "train_set,test_set = ud.get_mnist(conf)\n",
    "train_set = ud.get_augmented_dataset(conf, train_set,transform)\n",
    "test_set = ud.get_augmented_dataset(conf, test_set,transform)\n",
    "train_loader, valid_loader, test_loader = ud.train_valid_test_split(conf, train_set, test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tracked = ['loss', 'node_sparse']\n",
    "train_hist = {}\n",
    "val_hist = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 0\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 30.049007538706064\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.14264455782312926\n",
      "Overall sparsity: 0.14264455782312926\n",
      "Node sparsity: [0.3048469387755102, 0.09438775510204081, 0.008928571428571428, 0.008928571428571428, 0.016581632653061226, 0.4221938775510204]\n",
      "Regularization values per group: [940.5499606323243, 0.0, 0.0]\n",
      "tensor([0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 1\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 22.25697558745742\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.18345925742746078\n",
      "Overall sparsity: 0.18345925742746078\n",
      "Node sparsity: [0.3048469387755102, 0.15816326530612246, 0.007653061224489796, 0.01020408163265306, 0.02423469387755102, 0.5956632653061225]\n",
      "Regularization values per group: [1125.318122253418, 0.0, 0.0]\n",
      "tensor([0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([-0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., -0., 0., -0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 2\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 20.75221025198698\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.20726444276690267\n",
      "Overall sparsity: 0.20726444276690267\n",
      "Node sparsity: [0.32908163265306123, 0.21173469387755103, 0.007653061224489796, 0.01403061224489796, 0.044642857142857144, 0.6364795918367347]\n",
      "Regularization values per group: [1247.1098892211915, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([0., 0., -0.])\n",
      "tensor([-0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 3\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 19.910048689693213\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.22617718962585034\n",
      "Overall sparsity: 0.22617718962585034\n",
      "Node sparsity: [0.350765306122449, 0.24489795918367346, 0.007653061224489796, 0.029336734693877552, 0.07908163265306123, 0.6454081632653061]\n",
      "Regularization values per group: [1353.3837608337406, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([-0., -0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 4\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 18.290899395942688\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.22597056998819937\n",
      "Overall sparsity: 0.22597056998819937\n",
      "Node sparsity: [0.34566326530612246, 0.25255102040816324, 0.01403061224489796, 0.042091836734693876, 0.05102040816326531, 0.6505102040816326]\n",
      "Regularization values per group: [1443.0809165954593, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([-0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 5\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 17.608196143060923\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2321366205921144\n",
      "Overall sparsity: 0.2321366205921144\n",
      "Node sparsity: [0.34183673469387754, 0.29081632653061223, 0.01913265306122449, 0.044642857142857144, 0.04591836734693878, 0.6505102040816326]\n",
      "Regularization values per group: [1499.7263177490236, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([-0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 6\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 17.286323197185993\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.24021512303901152\n",
      "Overall sparsity: 0.24021512303901152\n",
      "Node sparsity: [0.3469387755102041, 0.3239795918367347, 0.02423469387755102, 0.05102040816326531, 0.044642857142857144, 0.6505102040816326]\n",
      "Regularization values per group: [1558.0447441101076, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 7\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 17.034898094832897\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.24850512590240179\n",
      "Overall sparsity: 0.24850512590240179\n",
      "Node sparsity: [0.3622448979591837, 0.34566326530612246, 0.03443877551020408, 0.04974489795918367, 0.04846938775510204, 0.6505102040816326]\n",
      "Regularization values per group: [1616.001719207764, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 8\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 16.742307670414448\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.25126818903581843\n",
      "Overall sparsity: 0.25126818903581843\n",
      "Node sparsity: [0.35586734693877553, 0.35714285714285715, 0.044642857142857144, 0.04846938775510204, 0.05102040816326531, 0.6505102040816326]\n",
      "Regularization values per group: [1685.2612483215332, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 9\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 16.212215032428503\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2542441083576288\n",
      "Overall sparsity: 0.2542441083576288\n",
      "Node sparsity: [0.3622448979591837, 0.3622448979591837, 0.05229591836734694, 0.04591836734693878, 0.05229591836734694, 0.6505102040816326]\n",
      "Regularization values per group: [1741.0002885437013, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 10\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 15.596826646476984\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2567948576114119\n",
      "Overall sparsity: 0.2567948576114119\n",
      "Node sparsity: [0.3647959183673469, 0.3635204081632653, 0.058673469387755105, 0.04974489795918367, 0.05357142857142857, 0.6505102040816326]\n",
      "Regularization values per group: [1801.756153869629, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., 0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 11\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 14.965143468230963\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2616791614605026\n",
      "Overall sparsity: 0.2616791614605026\n",
      "Node sparsity: [0.3737244897959184, 0.3647959183673469, 0.05739795918367347, 0.04591836734693878, 0.0778061224489796, 0.6505102040816326]\n",
      "Regularization values per group: [1844.9762832641604, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 12\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 14.49029910005629\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2633809263501319\n",
      "Overall sparsity: 0.2633809263501319\n",
      "Node sparsity: [0.375, 0.3673469387755102, 0.07015306122448979, 0.044642857142857144, 0.07270408163265306, 0.6505102040816326]\n",
      "Regularization values per group: [1898.6019442749025, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 13\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 14.100847387686372\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2689035276100236\n",
      "Overall sparsity: 0.2689035276100236\n",
      "Node sparsity: [0.3788265306122449, 0.3647959183673469, 0.08163265306122448, 0.04336734693877551, 0.09438775510204081, 0.6505102040816326]\n",
      "Regularization values per group: [1952.655099182129, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 14\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 13.777429681271315\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2710291067957795\n",
      "Overall sparsity: 0.2710291067957795\n",
      "Node sparsity: [0.3852040816326531, 0.3635204081632653, 0.09183673469387756, 0.039540816326530615, 0.09566326530612244, 0.6505102040816326]\n",
      "Regularization values per group: [2011.7942172241217, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 15\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 13.41482674330473\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2754939348188255\n",
      "Overall sparsity: 0.2754939348188255\n",
      "Node sparsity: [0.3877551020408163, 0.3635204081632653, 0.11224489795918367, 0.04591836734693878, 0.09311224489795919, 0.6505102040816326]\n",
      "Regularization values per group: [2075.573015289307, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 16\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 12.992942936718464\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2805957044807719\n",
      "Overall sparsity: 0.2805957044807719\n",
      "Node sparsity: [0.3877551020408163, 0.3647959183673469, 0.11607142857142858, 0.07015306122448979, 0.09438775510204081, 0.6505102040816326]\n",
      "Regularization values per group: [2121.1553894042972, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 17\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 12.422929326072335\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.28464241244967375\n",
      "Overall sparsity: 0.28464241244967375\n",
      "Node sparsity: [0.39413265306122447, 0.35841836734693877, 0.13010204081632654, 0.08035714285714286, 0.09438775510204081, 0.6505102040816326]\n",
      "Regularization values per group: [2170.166686859131, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 18\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 11.824478439986706\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.2899570383000139\n",
      "Overall sparsity: 0.2899570383000139\n",
      "Node sparsity: [0.399234693877551, 0.35459183673469385, 0.14923469387755103, 0.09056122448979592, 0.09566326530612244, 0.6505102040816326]\n",
      "Regularization values per group: [2217.904340209961, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 19\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 11.139057990163565\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.29293322877620437\n",
      "Overall sparsity: 0.29293322877620437\n",
      "Node sparsity: [0.40433673469387754, 0.35714285714285715, 0.14923469387755103, 0.09948979591836735, 0.09693877551020408, 0.6505102040816326]\n",
      "Regularization values per group: [2264.161884918213, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 20\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 10.539871696382761\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.29633458932042206\n",
      "Overall sparsity: 0.29633458932042206\n",
      "Node sparsity: [0.40816326530612246, 0.36096938775510207, 0.14923469387755103, 0.11224489795918367, 0.09693877551020408, 0.6505102040816326]\n",
      "Regularization values per group: [2307.6386631774903, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 21\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 10.05631392635405\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.30079887503470776\n",
      "Overall sparsity: 0.30079887503470776\n",
      "Node sparsity: [0.41836734693877553, 0.36096938775510207, 0.15306122448979592, 0.125, 0.09693877551020408, 0.6505102040816326]\n",
      "Regularization values per group: [2349.6036460876467, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 22\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 9.614788960665464\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.30334989544287105\n",
      "Overall sparsity: 0.30334989544287105\n",
      "Node sparsity: [0.41709183673469385, 0.3622448979591837, 0.1556122448979592, 0.1377551020408163, 0.09693877551020408, 0.6505102040816326]\n",
      "Regularization values per group: [2388.7517047119145, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 23\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 9.213417233899236\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3063260859190615\n",
      "Overall sparsity: 0.3063260859190615\n",
      "Node sparsity: [0.4247448979591837, 0.3622448979591837, 0.15943877551020408, 0.14285714285714285, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2425.035032348633, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 24\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 8.851575365290046\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.308239351225184\n",
      "Overall sparsity: 0.308239351225184\n",
      "Node sparsity: [0.42346938775510207, 0.3647959183673469, 0.15816326530612246, 0.15433673469387754, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2461.918807373047, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 25\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 8.534112127497792\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.31249105190545606\n",
      "Overall sparsity: 0.31249105190545606\n",
      "Node sparsity: [0.4362244897959184, 0.36989795918367346, 0.16198979591836735, 0.15816326530612246, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2495.9114059448248, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 26\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 8.244963441044092\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.31482948727960575\n",
      "Overall sparsity: 0.31482948727960575\n",
      "Node sparsity: [0.44260204081632654, 0.36862244897959184, 0.16326530612244897, 0.16581632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2529.861117553711, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 27\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 7.960333173163235\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.31950635802790506\n",
      "Overall sparsity: 0.31950635802790506\n",
      "Node sparsity: [0.4502551020408163, 0.3762755102040816, 0.17091836734693877, 0.17091836734693877, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2561.0765011596677, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 28\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 7.704897568561137\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.32269513353810914\n",
      "Overall sparsity: 0.32269513353810914\n",
      "Node sparsity: [0.45408163265306123, 0.38010204081632654, 0.17729591836734693, 0.1760204081632653, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2592.3605218505863, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 29\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 7.467799426987767\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.326521664150354\n",
      "Overall sparsity: 0.326521664150354\n",
      "Node sparsity: [0.45790816326530615, 0.3864795918367347, 0.18494897959183673, 0.18112244897959184, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2624.709179992676, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 30\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 7.248032757081091\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3303481947625989\n",
      "Overall sparsity: 0.3303481947625989\n",
      "Node sparsity: [0.45790816326530615, 0.39668367346938777, 0.18622448979591838, 0.19260204081632654, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2653.6122265625004, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 31\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 7.020817885175347\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.333536970272803\n",
      "Overall sparsity: 0.333536970272803\n",
      "Node sparsity: [0.4642857142857143, 0.4017857142857143, 0.1913265306122449, 0.1951530612244898, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2683.4624798583986, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 32\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 6.794962546788156\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.33545023557892545\n",
      "Overall sparsity: 0.33545023557892545\n",
      "Node sparsity: [0.4655612244897959, 0.4119897959183674, 0.1875, 0.1989795918367347, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2710.2721252441406, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 33\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 6.581564664840698\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3380012559870887\n",
      "Overall sparsity: 0.3380012559870887\n",
      "Node sparsity: [0.46938775510204084, 0.41964285714285715, 0.1913265306122449, 0.1989795918367347, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2736.5577212524418, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., 0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 34\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 6.369221194647253\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3409774464632792\n",
      "Overall sparsity: 0.3409774464632792\n",
      "Node sparsity: [0.4732142857142857, 0.42346938775510207, 0.1989795918367347, 0.20153061224489796, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2764.13201965332, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 35\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 6.181030251085758\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3428907117694016\n",
      "Overall sparsity: 0.3428907117694016\n",
      "Node sparsity: [0.4744897959183674, 0.42857142857142855, 0.2002551020408163, 0.20535714285714285, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2791.181969604492, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 36\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 5.99550571013242\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3452291471435513\n",
      "Overall sparsity: 0.3452291471435513\n",
      "Node sparsity: [0.475765306122449, 0.43494897959183676, 0.20408163265306123, 0.20790816326530612, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2819.9959524536134, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 37\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 5.825928448699415\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3462920723136193\n",
      "Overall sparsity: 0.3462920723136193\n",
      "Node sparsity: [0.475765306122449, 0.4362244897959184, 0.20535714285714285, 0.21173469387755103, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2848.416879577637, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 38\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 5.665043592453003\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3488430927217826\n",
      "Overall sparsity: 0.3488430927217826\n",
      "Node sparsity: [0.4770408163265306, 0.44005102040816324, 0.21301020408163265, 0.21428571428571427, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2877.085707702637, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 39\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 5.507173804566264\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.35075635802790506\n",
      "Overall sparsity: 0.35075635802790506\n",
      "Node sparsity: [0.4770408163265306, 0.4387755102040816, 0.22321428571428573, 0.21683673469387754, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2904.347834472657, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 40\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 5.363917821086943\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3526696233340275\n",
      "Overall sparsity: 0.3526696233340275\n",
      "Node sparsity: [0.47831632653061223, 0.4387755102040816, 0.23341836734693877, 0.21683673469387754, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2931.657438964844, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 41\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 5.227311415597796\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3543703036061363\n",
      "Overall sparsity: 0.3543703036061363\n",
      "Node sparsity: [0.47959183673469385, 0.4387755102040816, 0.24107142857142858, 0.2181122448979592, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2955.153662719727, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 42\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 5.106233164668083\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.35500805870817714\n",
      "Overall sparsity: 0.35500805870817714\n",
      "Node sparsity: [0.48086734693877553, 0.4387755102040816, 0.24362244897959184, 0.2181122448979592, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [2979.299291992188, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 43\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.9965724386274815\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.35607098387824515\n",
      "Overall sparsity: 0.35607098387824515\n",
      "Node sparsity: [0.48086734693877553, 0.44260204081632654, 0.2461734693877551, 0.2181122448979592, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3002.9838864135745, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 44\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.894682862795889\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3569213240142996\n",
      "Overall sparsity: 0.3569213240142996\n",
      "Node sparsity: [0.48086734693877553, 0.4451530612244898, 0.24744897959183673, 0.2193877551020408, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3025.1745700073247, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 45\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.810959783382714\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.35755907911634044\n",
      "Overall sparsity: 0.35755907911634044\n",
      "Node sparsity: [0.48086734693877553, 0.44642857142857145, 0.25, 0.2193877551020408, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3048.2813418579103, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 46\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.722239773720503\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.35755907911634044\n",
      "Overall sparsity: 0.35755907911634044\n",
      "Node sparsity: [0.48086734693877553, 0.44642857142857145, 0.25, 0.2193877551020408, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3070.0536981201176, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 47\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.640885388478637\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3581968342183812\n",
      "Overall sparsity: 0.3581968342183812\n",
      "Node sparsity: [0.48086734693877553, 0.4477040816326531, 0.25255102040816324, 0.2193877551020408, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3093.0948236083987, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 48\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.569163073785603\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.35904717435443567\n",
      "Overall sparsity: 0.35904717435443567\n",
      "Node sparsity: [0.48086734693877553, 0.4502551020408163, 0.25510204081632654, 0.2193877551020408, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3115.8023489379884, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 49\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.49493679124862\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.35968492945647645\n",
      "Overall sparsity: 0.35968492945647645\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.2576530612244898, 0.2193877551020408, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3136.876192016602, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 50\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.424486213363707\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3601100995245037\n",
      "Overall sparsity: 0.3601100995245037\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.2602040816326531, 0.2193877551020408, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3157.71125579834, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 51\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.3568358374759555\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3603226845585173\n",
      "Overall sparsity: 0.3603226845585173\n",
      "Node sparsity: [0.48086734693877553, 0.4502551020408163, 0.2614795918367347, 0.22066326530612246, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3179.2355465698247, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 52\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.295875800773501\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3601100995245037\n",
      "Overall sparsity: 0.3601100995245037\n",
      "Node sparsity: [0.48086734693877553, 0.4489795918367347, 0.2614795918367347, 0.22066326530612246, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3200.02839630127, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 53\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.232708620838821\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3611730246945717\n",
      "Overall sparsity: 0.3611730246945717\n",
      "Node sparsity: [0.48086734693877553, 0.4489795918367347, 0.2653061224489796, 0.22321428571428573, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3220.537172241211, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 54\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.181098408997059\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3615981947625989\n",
      "Overall sparsity: 0.3615981947625989\n",
      "Node sparsity: [0.48086734693877553, 0.4502551020408163, 0.26658163265306123, 0.22321428571428573, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3240.996018066406, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 55\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.12702779378742\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3618107797966125\n",
      "Overall sparsity: 0.3618107797966125\n",
      "Node sparsity: [0.48086734693877553, 0.4502551020408163, 0.26785714285714285, 0.22321428571428573, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3261.736201171875, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 56\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.088464874774218\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36223594986463975\n",
      "Overall sparsity: 0.36223594986463975\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.26913265306122447, 0.22321428571428573, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3281.6004614257813, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 57\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 4.034041568636894\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36223594986463975\n",
      "Overall sparsity: 0.36223594986463975\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.26785714285714285, 0.22448979591836735, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3301.446657714844, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 58\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.982680437620729\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36308629000069415\n",
      "Overall sparsity: 0.36308629000069415\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.27168367346938777, 0.22448979591836735, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3320.65361694336, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 59\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.94175515929237\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36287370496668053\n",
      "Overall sparsity: 0.36287370496668053\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.27168367346938777, 0.22448979591836735, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3340.1097332763675, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 60\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.9106878712773323\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36308629000069415\n",
      "Overall sparsity: 0.36308629000069415\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.2729591836734694, 0.22448979591836735, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3358.949401855469, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 61\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.857318284921348\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36287370496668053\n",
      "Overall sparsity: 0.36287370496668053\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.2729591836734694, 0.22321428571428573, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3378.3413671875005, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 62\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.821616572327912\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36287370496668053\n",
      "Overall sparsity: 0.36287370496668053\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.2729591836734694, 0.22321428571428573, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3397.5572088623053, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 63\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.7810464655049145\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36308629000069415\n",
      "Overall sparsity: 0.36308629000069415\n",
      "Node sparsity: [0.48086734693877553, 0.45153061224489793, 0.2729591836734694, 0.22448979591836735, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3415.8378216552737, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 64\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.7473465907387435\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.363724045102735\n",
      "Overall sparsity: 0.363724045102735\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.274234693877551, 0.22576530612244897, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3434.4271069335937, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 65\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.7138990499079227\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3641492151707622\n",
      "Overall sparsity: 0.3641492151707622\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.2767857142857143, 0.22576530612244897, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3451.6563012695315, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 66\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.6785215428099036\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3645743852387894\n",
      "Overall sparsity: 0.3645743852387894\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.27933673469387754, 0.22576530612244897, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3469.376303100586, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 67\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.6433018539100885\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3645743852387894\n",
      "Overall sparsity: 0.3645743852387894\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.27933673469387754, 0.22576530612244897, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3486.634537353516, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 68\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.608639905229211\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.364786970272803\n",
      "Overall sparsity: 0.364786970272803\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22576530612244897, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3503.980728759766, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 69\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.5776695273816586\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.364786970272803\n",
      "Overall sparsity: 0.364786970272803\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22576530612244897, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3521.1000738525395, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 70\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.5495173246599734\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3652121403408302\n",
      "Overall sparsity: 0.3652121403408302\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22831632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3538.724851074219, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 71\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.5198562094010413\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3652121403408302\n",
      "Overall sparsity: 0.3652121403408302\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22831632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3555.2128247070314, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 72\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.493012249004096\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3652121403408302\n",
      "Overall sparsity: 0.3652121403408302\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22831632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3572.813646240235, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 73\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.464361562859267\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3652121403408302\n",
      "Overall sparsity: 0.3652121403408302\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22831632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3588.7492633056645, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 74\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.431485458277166\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3652121403408302\n",
      "Overall sparsity: 0.3652121403408302\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22831632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3605.4590588378915, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 75\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.405869649723172\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3622.4295257568365, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 76\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.388988295570016\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3637.203516235352, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 77\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.366687349975109\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3653.7355432128916, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 78\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.3366099819540977\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3652121403408302\n",
      "Overall sparsity: 0.3652121403408302\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22831632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3668.7786688232427, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 79\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.3129487708210945\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.3652121403408302\n",
      "Overall sparsity: 0.3652121403408302\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22831632653061223, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3684.688416137696, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 80\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.288967465981841\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3701.1261157226572, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 81\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.275084322784096\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3716.5131744384767, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 82\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.2501525706611574\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3731.878909301758, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 83\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.2243181564845145\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3747.6155535888674, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 84\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.208720825612545\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3763.0009375000004, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 85\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.189438214059919\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3777.3273663330083, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 86\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.1737921787425876\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3792.673033447266, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 87\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.1555459196679294\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3807.6213830566417, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 88\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.13422342389822\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3821.5518402099615, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 89\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.1177230053581297\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3836.3290307617194, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 90\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.1055162819102407\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36563731040885744\n",
      "Overall sparsity: 0.36563731040885744\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.2308673469387755, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3850.0528887939454, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 91\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.0892296908423305\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3863.740260009766, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 92\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.0688667688518763\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36563731040885744\n",
      "Overall sparsity: 0.36563731040885744\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.2308673469387755, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3878.4547650146487, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 93\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.059598707128316\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3893.03956237793, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 94\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.0376089941710234\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3906.840401611328, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 95\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.024018364492804\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3920.4475915527346, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 96\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 3.018374747596681\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3932.9304791259774, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 97\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 2.991888353601098\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3945.3468829345707, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 98\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 2.9858967941254377\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3958.828073730469, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 99\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.0\n",
      "Train Loss: 2.967544010374695\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.0\n",
      "Non-zero kernels: 0\n",
      "Linear sparsity: 0.36542472537484383\n",
      "Overall sparsity: 0.36542472537484383\n",
      "Node sparsity: [0.48086734693877553, 0.4528061224489796, 0.28061224489795916, 0.22959183673469388, 0.09821428571428571, 0.6505102040816326]\n",
      "Regularization values per group: [3971.967215576172, 0.0, 0.0]\n",
      "tensor([-0.])\n",
      "tensor([-0., -0.])\n",
      "tensor([-0., -0., -0.])\n",
      "tensor([-0., 0., -0., -0.])\n",
      "tensor([0., 0., -0., -0., -0.])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "Learning rate: 0.001\n"
     ]
    }
   ],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# Reinit weigts and the corresponding optimizer\n",
    "# -----------------------------------------------------------------------------------\n",
    "model = init_weights(conf, model)\n",
    "opt, scheduler = init_opt(conf, model)\n",
    "\n",
    "# -----------------------------------------------------------------------------------\n",
    "# train the model\n",
    "# -----------------------------------------------------------------------------------\n",
    "for epoch in range(conf.epochs):\n",
    "    print(25*\"<>\")\n",
    "    print(50*\"|\")\n",
    "    print(25*\"<>\")\n",
    "    print('Epoch:', epoch)\n",
    "\n",
    "    # ------------------------------------------------------------------------\n",
    "    # train step, log the accuracy and loss\n",
    "    # ------------------------------------------------------------------------\n",
    "    train_data = train.train_step(conf, model, opt, train_loader)\n",
    "\n",
    "    # update history\n",
    "    for key in tracked:\n",
    "        if key in train_data:\n",
    "            var_list = train_hist.setdefault(key, [])\n",
    "            var_list.append(train_data[key])        \n",
    "\n",
    "    # ------------------------------------------------------------------------\n",
    "    # validation step\n",
    "    val_data = train.validation_step(conf, model, opt, valid_loader)\n",
    "\n",
    "    print_skips(model)\n",
    "    # update history\n",
    "    for key in tracked:\n",
    "        if key in val_data:\n",
    "            var = val_data[key]\n",
    "            if isinstance(var, list):\n",
    "                for i, var_loc in enumerate(var):\n",
    "                    key_loc = key+\"_\" + str(i)\n",
    "                    var_list = val_hist.setdefault(key_loc, [])\n",
    "                    val_hist[key_loc].append(var_loc)\n",
    "            else:\n",
    "                var_list = val_hist.setdefault(key, [])\n",
    "                var_list.append(var)\n",
    "\n",
    "\n",
    "    scheduler.step(train_data['loss'])\n",
    "    print(\"Learning rate:\",opt.param_groups[0]['lr'])\n",
    "    best_model(train_data['acc'], val_data['acc'], model=model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "07cecbd58096559357ff47686066ecdb82e32493890ce712e45aa4f3fdd673c4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
